#!/usr/bin/perl

#
# sefapi - SEarch Flickr API
#          http://code.google.com/p/sefapi/
#          http://www.apache.org/licenses/LICENSE-2.0
#          http://creativecommons.org/licenses/by-sa/3.0/
#
$main::VERSION = "1.1.0";

#
# This script expects you to have a configuration rc file at ~/.sefapirc.  This location can
# be overridden with the -rc_file option.  The file should have the following three lines:
#
#   flickr_key=<application_key>
#   flickr_secret=<application_secret>
#   flickr_auth_token=<token>
#
# For more information on what those things are, see http://www.flickr.com/services/api/
#

#
# Sample Usage:
#
# Find the URLs of 15 5-or-more-megapixel pictures of dirt roads:
#
#   sefapi.pl -max-results=15 -text="dirt road" -min-size=5
#
# That search will return logs of 'large' images but few 'original' images.
# To fetch the full-sized originals of all pics that are portrait add a couple options:
#
#   sefapi.pl -max-results=15 -text="dirt road" -min-size=5 \
#             -suffix=o -orientation=tall
#
# To constrain the search to only images tagged as 'bike' or 'mud' add another option:
#
#   sefapi.pl -max-results=15 -text="dirt road" -min-size=5 \
#           -suffix=o -orientation=tall \
#           -tags=bike,mud
#
# To constrain the search to only images tagged as 'bike' AND 'mud' tweak it a bit:
#
#   sefapi.pl -max-results=15 -text="dirt road" -min-size=5 \
#           -suffix=o -orientation=tall \
#           -tags=bike,mud -tag-mode=all
#
# So, to look for astronomy wallpapers you might use this:
#
#   sefapi.pl -max-results=10 -tags=messier,astronomy -tag-mode=all -min-size=10
#             -orientation=wide -suffix=o
#
# And if you just want 10 pictures of dogs?
#
#   sefapi.pl -max-results=10 -tags=dog
#
# Or the newest or oldest 10 pictures of dogs?
#
#   sefapi.pl -max-results=10 -tags=dog -sort=date-posted-desc -action=download
#   sefapi.pl -max-results=10 -tags=dog -sort=date-posted-asc  -action=download
#
# But most of those old images are only available in their small versions:
#
#   sefapi.pl -max-results=10 -tags=dog -sort=date-posted-asc -suffix=m -action=download
#
# And if you're searching for dogs you probably want the best dogs
#
#   sefapi.pl -max-results=10 -tags=dog -sort=date-posted-asc -suffix=m \
#             -text="chesapeake bay retriever" -action=download
#

use Config::General qw/ParseConfig/;
use Data::Dumper;
use Flickr::API;
use Getopt::Long qw(:config auto_version);
use Pod::Usage;
use XML::Parser::Lite::Tree::XPath;
use Term::ProgressBar;
use Image::MetaData::JPEG;
$JPEG::show_warnings = undef;
use Statistics::Descriptive::Discrete;
use Text::Levenshtein qw(distance);

use strict;
use warnings;

pod2usage(-verbose => 1) if @ARGV == 0;

my $user_agent = LWP::UserAgent->new;
my $user_agent_string = "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.0.10) Gecko/2009042315 Firefox/3.0.10";
$user_agent->agent($user_agent_string);
$user_agent->max_redirect(0);

my ($applescript) = $0 =~ /(^.*)\//;
$applescript .= "/iphoto-import.applescript";

#
# Parse options
#
my $opt_action = "download";
my $opt_aspect_ratio;
my $opt_debug = 0;
my $opt_directory;
my $opt_exclude_tags;
my $opt_group;
my $opt_help = 0;
my $opt_images_per_page;
my $opt_iphoto_import;
my $opt_max_results = 25;
my $opt_media = "all";
my $opt_min_size = -1;
my $opt_orientation;
my $opt_photoset;
my $opt_privacy_filter = 1;
my $opt_rc_file = $ENV{"HOME"}."/.sefapirc";
my $opt_sort = "interestingness-desc";
my $opt_suffix = "b";
my $opt_tag_mode;
my $opt_tags = undef;
my $opt_text;
my $opt_user;
my $opt_verbose = 2;
my $opt_verbose_log = 3;

GetOptions(
           "action=s" => \$opt_action,
           "aspect-ratio=s" => \$opt_aspect_ratio,
           "debug" => \$opt_debug,
           "directory" => \$opt_directory,
           "exclude-tags=s" => \$opt_exclude_tags,
           "group=s" => \$opt_group,
           "help" => \$opt_help,
           "images-per-page=i" => \$opt_images_per_page,
           "iphoto-import" => \$opt_iphoto_import,
           "max-results=i" => \$opt_max_results,
           "media=s" => \$opt_media,
           "min-size=i" => \$opt_min_size,
           "orientation=s" => \$opt_orientation,
           "photoset=s" => \$opt_photoset,
           "privacy-filter=i" => \$opt_privacy_filter,
           "rc-file=s" => \$opt_rc_file,
           "sort=s" => \$opt_sort,
           "suffix=s" => \$opt_suffix,
           "tag-mode=s" => \$opt_tag_mode,
           "tags=s" => \$opt_tags,
           "text=s" => \$opt_text,
           "user=s" => \$opt_user,
           "verbose+" => \$opt_verbose,
           "verbose-log+" => \$opt_verbose_log,
          ) or pod2usage(1);
pod2usage(-verbose => 1) if $opt_help;


#
# Load and check configuration
#
my %sefapirc = ParseConfig($opt_rc_file);
{
  my $required_config_keys = [ "flickr_key", "flickr_secret" ];
  map { pod2usage("missing $_ configuration key") unless defined $sefapirc{$_} } @$required_config_keys;
}

# working directory is where we'll download pictures and store logs
my $working_directory = $sefapirc{directory};
$working_directory = $opt_directory if defined $opt_directory;
$working_directory = "./sefapi" unless defined $working_directory;

# where we'll download images to
# imports to iphoto will be placed in an event named for this directory
my $download_directory;
{
  my $timestamp = `date +%s`;
  chomp $timestamp;
  $working_directory = "$working_directory/$timestamp";
  $download_directory = "$working_directory/files";
}


# directories that must exist
my $required_directories = [ $download_directory ];
for my $directory (@$required_directories) {
  die "directory already existed: $directory" if -e $directory;
  `mkdir -p $directory`;
  die "could not find or make directory: $directory" unless -e $directory;
}

my $log_file = "$working_directory/sefapi.log";

#
#  Construct global Flickr API object
#
my $api = new Flickr::API({
                           key => $sefapirc{flickr_key},
                           secret => $sefapirc{flickr_secret},
                          });

#
# Construct search parameters common to each search call
#
my %search_parameters;
{

  if (defined $opt_group) {
    # the following are used for group searches
    my ($group_id) = $opt_group =~ m/([0-9]+\@N[0-9]{2})/;
    die "couldn't find a group id in $opt_group" unless defined $group_id;
    print "group id: $group_id\n";
    # group_id (Required)
    #     The id of the group who's pool you which to get the photo list for.
    $search_parameters{group_id} = $group_id;
  }
  elsif (defined $opt_photoset) {
    # the following are used for photoset searches
    my $photoset_url = $opt_photoset;
    my ($photoset_id) = $opt_photoset =~ m/([0-9]{6,})/;
    $search_parameters{photoset_id} = $photoset_id;
    print "photoset id: $photoset_id\n";
  }
  else {
    # the following are used for user, tag or text searches

    #   user_id (Optional)
    #       The NSID of the user who's photo to search. If this parameter isn't passed then everybody's public photos will be searched. A value of "me" will search against the calling user's photos for authenticated calls.
    ($search_parameters{user_id}) = get_user_info($opt_user) if defined $opt_user;

    #   tags (Optional)
    #       A comma-delimited list of tags. Photos with one or more of the tags listed will be returned.
    $search_parameters{tags} = $opt_tags if defined $opt_tags;

    #   tag_mode (Optional)
    #       Either 'any' for an OR combination of tags, or 'all' for an AND combination. Defaults to 'any' if not specified.
    $search_parameters{tag_mode} = $opt_tag_mode if defined $opt_tag_mode;
    $search_parameters{tag_mode} = "all" if defined $opt_tags && !defined $search_parameters{tag_mode};

    #   text (Optional)
    #       A free text search. Photos who's title, description or tags contain the text will be returned.
    $search_parameters{text} = $opt_text if defined $opt_text;

    #   machine_tags (Optional)
    #       Aside from passing in a fully formed machine tag, there is a special syntax for searching on specific properties :
    #           * Find photos using the 'dc' namespace : "machine_tags" => "dc:"
    #           * Find photos with a title in the 'dc' namespace : "machine_tags" => "dc:title="
    #           * Find photos titled "mr. camera" in the 'dc' namespace : "machine_tags" => "dc:title=\"mr. camera\"
    #           * Find photos whose value is "mr. camera" : "machine_tags" => "*:*=\"mr. camera\""
    #           * Find photos that have a title, in any namespace : "machine_tags" => "*:title="
    #           * Find photos that have a title, in any namespace, whose value is "mr. camera" : "machine_tags" => "*:title=\"mr. camera\""
    #           * Find photos, in the 'dc' namespace whose value is "mr. camera" : "machine_tags" => "dc:*=\"mr. camera\""
    #       Multiple machine tags may be queried by passing a comma-separated list. The number of machine tags you can pass in a single query depends on the tag mode (AND or OR) that you are querying with. "AND" queries are limited to (16) machine tags. "OR" queries are limited to (8).

    #   machine_tag_mode (Required)
    #       Either 'any' for an OR combination of tags, or 'all' for an AND combination. Defaults to 'any' if not specified.

    #   content_type (Optional)
    #       Content Type setting:
    #          * 1 for photos only.
    #          * 2 for screenshots only.
    #          * 3 for 'other' only.
    #          * 4 for photos and screenshots.
    #          * 5 for screenshots and 'other'.
    #          * 6 for photos and 'other'.
    #          * 7 for photos, screenshots, and 'other' (all).

    #   safe_search (Optional)
    #       Safe search setting:
    #           * 1 for safe.
    #           * 2 for moderate.
    #           * 3 for restricted.
    $search_parameters{safe_search} = 3;

    #   sort (Optional)
    #       The order in which to sort returned photos. Deafults to date-posted-desc (unless you are doing a radial geo query...)
    #       The possible values are:
    #            * date-posted-asc
    #            * date-posted-desc
    #            * date-taken-asc
    #            * date-taken-desc
    #            * interestingness-desc
    #            * interestingness-asc
    #            * relevance
    $search_parameters{sort} = $opt_sort;
  }

  # all of the following are common to photoset, tag and text searches
  {
    #   media (Optional)
    #       Filter results by media type. Possible values are all (default), photos or videos
    $search_parameters{media} = $opt_media if defined $opt_media;

    #   per_page (Optional)
    #       Number of photos to return per page. If this argument is omitted, it defaults to 100. The maximum allowed value is 500.
    $search_parameters{per_page} = $opt_images_per_page;
    $search_parameters{per_page} = $opt_max_results < 0 || $opt_max_results > 50 ? 500 : 100 unless defined $search_parameters{per_page};

    #   privacy_filter (Optional)
    #       Return photos only matching a certain privacy level. This only applies when making an authenticated call to view photos you own. Valid values are:
    #           * 1 public photos
    #           * 2 private photos visible to friends
    #           * 3 private photos visible to family
    #           * 4 private photos visible to friends & family
    #           * 5 completely private photos
    $search_parameters{privacy_filter} = $opt_privacy_filter;

    #  extras (Optional)
    #      A comma-delimited list of extra information to fetch for each returned record.
    #      Currently supported fields are: license, date_upload, date_taken, owner_name, icon_server,
    #      original_format, last_update, geo, tags, machine_tags, o_dims, views, media.
    $search_parameters{extras} = "o_dims,tags,owner_name";

    $search_parameters{api_key}    = $sefapirc{flickr_key};
    $search_parameters{auth_token} = $sefapirc{flickr_auth_token};
  }
}

my @excluded_tags;
if (defined $opt_exclude_tags) {
  @excluded_tags = split(/,/, $opt_exclude_tags);
}


open(LOG, ">>$log_file");
my $results = do_search();
action_download_images($results);
if ($opt_iphoto_import) {
  die "could not find required iphoto import script: $applescript" unless -e $applescript;
  `$applescript $download_directory`;
  print "spawned iphoto import process for $download_directory\n";
}
close(LOG);
print "download complete at $download_directory\n";




sub inform {
  my ($level_str, $message) = @_;
  die "missing message for $level_str\n" unless defined $message;
  my $verbosity_map = {
                       fatal => 0,
                       error => 1,
                       warn  => 2,
                       info  => 3,
                       debug => 4,
                      };
  my $level = $verbosity_map->{$level_str};
  die "invalid log level: '$level_str'" unless defined $level;
  my $prefix = "[".localtime()."] ";
  $message =~ s/\n/\n$prefix/g;
  if ($level <= $opt_verbose_log) {
    # strip non-ascii characters
    $message =~ s/[^[:ascii:]]//g;
    print LOG $prefix.$message."\n";
    flush LOG;
  }
  if ($level <= $opt_verbose) {
    print $prefix.$message."\n";
  }
}


sub do_search {
  inform("info", "searching:");
  map { inform("info", "\t$_=$search_parameters{$_}") unless $_ eq "auth_token" } sort keys %search_parameters;
  my $results = {};
  my $expected_count = $opt_max_results > 0 ? $opt_max_results : 1000;
  my $progress = Term::ProgressBar->new ({
                                          name => "searching",
                                          count => $expected_count,
                                          ETA   => "linear",
                                          term_width => 80,
                                          max_update_rate => .1,
                                         });
  for (my $page_number = 1 ; ; $page_number++) {
    inform("debug", "searching for page $page_number");
    my $page = search_photos(\%search_parameters, $page_number);
    inform("debug", "got page: $page_number with ".scalar(keys %$page)." results\n");
    goto end unless keys %$page; # we're out of results
    for (keys %$page) {
      if (should_be_filtered_out($page->{$_})) {
        next;
      }
      # if we've already seen this item then we're at the end of our results
      goto end if defined $results->{$_};
      $results->{$_} = $page->{$_};
      fill_in_details($results->{$_});
      $progress->update();
      goto end if $opt_max_results > 0 && keys %$results == $opt_max_results;
    }
  }
 end:
  $progress->update($opt_max_results);
  print "\n";
  return $results;
}



sub action_download_images {
  my ($images) = @_;
  my $wget_flags = "wget --user-agent='$user_agent_string' --no-clobber --quiet --directory-prefix $download_directory --output-file=$working_directory/wget.log ";
  my $progress = Term::ProgressBar->new ({
                                          name => "downloading",
                                          count => scalar(keys %$images),
                                          ETA   => "linear",
                                          term_width => 80,
                                          max_update_rate => .1,
                                         });
  my $downloaded = [];
  for (sort keys %$images) {
    my $resolution;
    if (defined $images->{$_}{o_width} && defined $images->{$_}{o_width}) {
      $resolution = sprintf("%sx%s (%s",
                            $images->{$_}{o_width},
                            $images->{$_}{o_height},
                            $images->{$_}{sefapi_aspect_ratio});
      $resolution .= ", $images->{$_}{sefapi_orientation}" if defined $images->{$_}{sefapi_orientation};
      $resolution .= ")";
    } else {
      $resolution = "unknown ";
    }
    my $filename = $images->{$_}{sefapi_photo_url};
    $filename =~ s/.*\///;
    my $info = <<END;
$images->{$_}{sefapi_photo_url}
 ... id: $images->{$_}{id}
 ... own: $images->{$_}{owner}
 ... res: $resolution
 ... tag: $images->{$_}{tags}
 ... ttl: $images->{$_}{title}
 ... web: $images->{$_}{sefapi_photo_page}
END
    `wget $wget_flags $images->{$_}{sefapi_photo_url}`;
    # tag the photo
    my $photo_path = "$download_directory/$filename";
    tag_image($photo_path, $images->{$_}{tags});
    push(@$downloaded, $photo_path);
    $progress->update();
    inform("info", $info);
  }
  $progress->update(scalar(keys %$images));
  print "\n";
  return $downloaded;
}


sub should_be_filtered_out {
  my ($pic) = @_;

  # size filter
  if ($opt_min_size > -1
      && (!defined $pic->{sefapi_size}
          || $pic->{sefapi_size} < $opt_min_size * 1000 * 1000)) {
    return 1;
  }

  # aspect ratio filter
  if (defined $opt_aspect_ratio) {
    if (defined $pic->{sefapi_aspect_ratio} && abs($opt_aspect_ratio - $pic->{sefapi_aspect_ratio}) < .1) {
      #      inform("info", "image passed aspect ratio filter: requested: $opt_aspect_ratio found: $pic->{sefapi_aspect_ratio} diff: ".(abs($opt_aspect_ratio - $pic->{sefapi_aspect_ratio})));
    } else {
      return 1;
    }
  }

  # orientation filter
  if (defined $opt_orientation) {
    return 1 unless defined $pic->{sefapi_aspect_ratio};
    return 1 if $opt_orientation eq "wide" && $pic->{sefapi_aspect_ratio} < 1.2;
    return 1 if $opt_orientation eq "tall" && $pic->{sefapi_aspect_ratio} > 0.8;
  }

  # tag filter
  map { return 1 if $pic->{tags} =~ /$_/ } @excluded_tags if defined $pic->{tags};

  # image exists
  # this requires the image url, which is calculated in fill_in_details()
  # this call may require a query of flickr, so we do every other filter first
  # in order to avoid the expensive round trip to flickr
  fill_in_details($pic);
  if (!defined $pic->{sefapi_photo_url}) {
    return 1;
  }

  # photo_unavailable.gif is what we get redirected to if the image no longer exists
  if (Dumper($user_agent->head($pic->{sefapi_photo_url})) =~ /photo_unavailable\.gif/) {
    return 1;
  }

  return 0;
}


# Read a configuration file
#   The arg can be a relative or full path, or
#   it can be a file located somewhere in @INC.
sub read_cfg
{
  my $file = $_[0];
  inform("debug", "read_cfg($file)");
  our $err;
  {                        # Put config data into a separate namespace
    package CFG;
    # Process the contents of the config file
    my $rc = do($file);
    # Check for errors
    if ($@) {
      $::err = "ERROR: Failure compiling '$file' - $@";
    } elsif (! defined($rc)) {
      $::err = "ERROR: Failure reading '$file' - $!";
    } elsif (! $rc) {
      $::err = "ERROR: Failure processing '$file'";
    }
  }
  return ($err);
}


sub search_photos
{
  my ($params, $page) = @_;
  inform("debug", "search_photos(<params>, page=$page)");
  my %p = %$params;
  $p{page} = $page;
  my $response;
  if (defined $p{group_id}) {
    # a group pool search
    $response = $api->execute_method('flickr.groups.pools.getPhotos', \%p);
  }
  elsif (defined $p{photoset_id}) {
    # a photoset search
    $response = $api->execute_method('flickr.photosets.getPhotos', \%p);
  }
  else {
    $response = $api->execute_method('flickr.photos.search', \%p);
  }
  my $photos = _extract_nodes($response->{tree}, "name", "photo", "id");
  for my $pid (keys %$photos) {
    my $pic = $photos->{$pid};
    if (defined $pic->{o_width} && defined $pic->{o_height}) {
      $pic->{sefapi_size} = $pic->{o_height} * $pic->{o_width};
      $pic->{sefapi_aspect_ratio} = sprintf("%1.2f", $pic->{o_width}/$pic->{o_height});
    }
  }
  return $photos;
}


sub fill_in_details {
  my ($pic) = @_;
  return if $pic->{sefapi_details_filled_in};
  inform("debug", "fill_in_details(<pic-".$pic->{id}.">)");

  # fetch additional information for this image if originals were requested
  if ($opt_suffix eq "o") {
    my $response = $api->execute_method("flickr.photos.getInfo",
                                        {
                                         photo_id => $pic->{id},
                                         auth_token => $sefapirc{flickr_auth_token},
                                        });
    my $info = _extract_nodes($response->{tree}, "name", "photo", "id");
    # write that info into the current pic's information
    map { $pic->{$_} = $info->{$pic->{id}}{$_} unless defined $pic->{$_} } keys %{$info->{$pic->{id}}};
  }

  # add the url for the image location, originals require a secret
  if (defined $pic->{originalsecret} && $opt_suffix eq "o") {
    $pic->{sefapi_photo_url} = sprintf("http://farm%s.static.flickr.com/%s/%s_%s_o.%s",
                                       $pic->{farm}, $pic->{server}, $pic->{id}, $pic->{originalsecret}, $pic->{originalformat});
  } else {
    # we don't have the original secret, so if "o" was requested fall back to the default "b"
    my $suffix = $opt_suffix eq "o" ? "b" : $opt_suffix;
    $pic->{sefapi_photo_url} = sprintf("http://farm%s.static.flickr.com/%s/%s_%s_$suffix.jpg",
                                       $pic->{farm}, $pic->{server}, $pic->{id}, $pic->{secret});
  }

  # add the url for the photo's webpage
  if (defined $pic->{ownername} && !defined $pic->{owner}) {
    my $owner = get_user_info($pic->{ownername});
    $pic->{owner} = $owner;
  }
  $pic->{sefapi_photo_page} = sprintf("http://www.flickr.com/photos/%s/%s", $pic->{owner}, $pic->{id});
  $pic->{sefapi_details_filled_in} = 1;
}


sub get_user_info
{
  my ($clue) = @_;
  inform("debug", "get_user_info($clue)");
  if ($clue =~ /\d+\@.\d\d/) {
    return ($clue);
  }
  die "please provide url or username" unless defined $clue;
  my $response = $api->execute_method('flickr.urls.lookupUser',
                                      {
                                       url => $clue, auth_token => $sefapirc{flickr_auth_token} });
  $response = $api->execute_method('flickr.people.findByUsername',
                                   {
                                    username => $clue, auth_token => $sefapirc{flickr_auth_token} }) unless $response->{success};
  die "failed: ".$response->{error_message} unless $response->{success};
  my $xpath = new XML::Parser::Lite::Tree::XPath($response->{tree});
  my $node = $xpath->select_nodes('/user')->[0];
  die "no user data returned" unless defined $node;
  return ( $node->{attributes}{id}, $node->{children}[1]{children}[0]{content} );
}


#
# write keyword tags into the jpeg data structure and save file
#
sub tag_image {
  my ($filename, $tags) = @_;
  unless (ref($tags)) {
    my @tags = split(/ /, $tags);
    @tags = grep {!/[^[:ascii:]]/} @tags;
    @tags = grep {!/[=:]/} @tags;
    $tags = \@tags;
  }
  $tags = find_interesting_tags($tags);

  {
    # we'll add some additional tags to the list based on how we're searching
    my %hash;
    $hash{flickr}++;
    $hash{sefapi}++;
    if (defined $opt_tags && defined $opt_tag_mode && $opt_tag_mode eq "all" ) {
      map { $hash{$_}++ } split(/,/, $opt_tags);
    }
    if (defined $opt_text) {
      map { $hash{$_}++ } split(/ /, $opt_text);
    }
    map { $hash{$_}++ } @$tags;
    my @keys = keys %hash;
    $tags = \@keys;
  }

  return unless defined $tags && scalar(@$tags) > 0;
  my $file = new Image::MetaData::JPEG($filename);
  if (!$file) {
    inform("error", Image::MetaData::JPEG::Error());
    return;
  }
  $file->drop_segments('APP13');
  my $segment = $file->provide_app13_segment('IPTC');
  $segment->set_app13_data({ Keywords => $tags }, 'ADD', 'IPTC');
  $file->drop_segments("APP2");
  my $tmp_filename = $filename.".tmp";
  if ($file->save($tmp_filename)) {
    `rm $filename`;
    `mv $tmp_filename $filename`;
  } else {
    `rm $tmp_filename`;
  }
}



#
# flickr has some over-tagged images ... and some that
# have huge long tags
#
# we'll try to weed out the worst of them, leaning
# toward keeping shorter (and hopefully simpler) tags
#
sub find_interesting_tags {
  my ($tags, $conf) = @_;
  return [] unless defined $tags && scalar(@$tags) > 0;
  $conf = {} unless defined $conf;
  my $conf_defaults = {
                       min_tag_length => 4,
                       max_tag_length => 16,
                       min_tag_distance => 2,
                       max_tag_count => 15,
                       allow_numbers => 0,
                      };
  map { $conf->{$_} = $conf_defaults->{$_} unless defined $conf->{$_} } keys %$conf_defaults;

  my $hash = {};

  my $length_stats = new Statistics::Descriptive::Discrete;
  map { $length_stats->add_data(length($_)) } @$tags;

  # build each tag's distance score
 WORD: for my $tag (@$tags) {
    my $length = length($tag);

    # filter out tags that are too short or long
    next WORD unless defined $length
    && $length >= $conf->{min_tag_length}
    && $length <= $conf->{max_tag_length};

    for (@$tags) {
      my $distance = distance($tag, $_);
      next if $distance == 0;
      next WORD if $distance <= $conf->{min_tag_distance} && length($_) < length($tag);
    }

    # not interested in numerical tags
    next if $tag =~ m/[0-9]/ && !$conf->{allow_numbers};
    my $distance_stats = new Statistics::Descriptive::Discrete;
    map { $distance_stats->add_data($_) } distance($tag, @$tags);
    $hash->{$tag} = sprintf("%3.5f", $distance_stats->mean() / ($length + $length_stats->mean()));
  }

  # de-emphasize tags that start or end with existing tags
  for my $first (keys %$hash) {
    for my $second (keys %$hash) {
      next if $first eq $second;
      $hash->{$first} += 1 if $first =~ m/^$second/ || $first =~ m/$second$/ || $first =~ m/$second/;
    }
  }

  # trim length if needed
  my @keys = sort {$hash->{$a} <=> $hash->{$b}} keys %$hash;
  if (scalar(@keys) > $conf->{max_tag_count}) {
    map { delete $hash->{$_} } @keys[$conf->{max_tag_count}..scalar(@keys)-1];
  }

  @keys = sort {$hash->{$a} <=> $hash->{$b}} keys %$hash;
  return \@keys;
}


# _extract_nodes($tree, "name", "photo", "id", {})
sub _extract_nodes {
  my ($tree, $node_name_attrib, $node_name, $result_key, $result) = @_;
  return unless ref $tree;
  $result = {} unless defined $result;
  if (ref $tree eq "ARRAY") {
    map { _extract_nodes($_, $node_name_attrib, $node_name, $result_key, $result) } @$tree;
  } elsif (defined $tree->{$node_name_attrib} && $tree->{$node_name_attrib} eq $node_name) {
    $result->{$tree->{attributes}{$result_key}} = $tree->{attributes};
  } else {
    map { _extract_nodes($tree->{$_}, $node_name_attrib, $node_name, $result_key, $result) } grep { ref $tree->{$_} } keys %$tree;
  }
  return $result;
}






__END__

=head1 sefapi [options]


=head1 SYNOPSIS

sefapi.pl [options]

=head1 OPTIONS

 -aspect-ratio=<float>

 -max-results=[num]
      maximum number of results, default 10

 -media=[photos|videos|all]
      defaults to 'photos'

 -min-size=[megapixels]

 -orientation=[tall|wide]

 -privacy-filter=[1-5]
       privacy level of image
          1 -> public (default)
          2 -> friends
          3 -> family
          4 -> friends & family
          5 -> private

 -rc-file=[file]
       specify the location of config file, default ~/.sefapirc

 -sort=[sort-by-field]
       recognized fields:
                 date-posted-asc
                 date-posted-desc
                 date-taken-asc
                 date-taken-desc
                 interestingness-asc
                 interestingness-desc (default)
                 relevance

 -suffix=[s|t|m|-|b]
      selects the size of image url to use - in some cases 'o' will not be available
      and we'll fall back to 'b'

      possible values:
          s -> small
          t -> thumbnail
          m -> small
          - -> medium
          b -> large [default]
          o -> original (if available)

 -tag-mode=[any|all]
      logical tag mode, any=OR, all=AND (default)

 -tags=[tag csv]
      csv of tags

 -text=[text]
      free form search text

 -user=[username|url]
      search for a single user's images
      user is found by username or by the url of a page they own

=head1 DESCRIPTION

=cut

